{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "\n",
    "from collections import defaultdict\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "train_df = pd.read_csv('train_sub_count.csv')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "test_df = pd.read_csv('test_sub_count.csv')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "count = 0\n",
    "feat={}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "dd = train_df['device_id'].value_counts()\n",
    "for xx in dd.keys():\n",
    "    feat['did_'+xx] = dd[xx]\n",
    "dd = train_df['device_ip'].value_counts()\n",
    "for xx in dd.keys():\n",
    "    feat['dip_'+xx] = dd[xx]\n",
    "dd = train_df['user_id'].value_counts()\n",
    "for xx in dd.keys():\n",
    "    feat['uid_'+xx] = dd[xx]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "dd = test_df['device_id'].value_counts()\n",
    "for xx in dd.keys():\n",
    "    feat['did_'+xx] = dd[xx]\n",
    "dd = test_df['device_ip'].value_counts()\n",
    "for xx in dd.keys():\n",
    "    feat['dip_'+xx] = dd[xx]\n",
    "dd = test_df['user_id'].value_counts()\n",
    "for xx in dd.keys():\n",
    "    feat['uid_'+xx] = dd[xx]    "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "import pickle"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "pickle.dump(feat, open(\"id_stat.pkl\", 'wb'))  "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "pickle.dump(feat, open(\"id_stat_test.pkl\", 'wb'))  "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "data": {
      "text/plain": "Index(['id', 'click', 'hour', 'C1', 'banner_pos', 'site_id', 'site_domain',\n       'site_category', 'app_id', 'app_domain', 'app_category', 'device_id',\n       'device_ip', 'device_model', 'device_type', 'device_conn_type', 'C14',\n       'C15', 'C16', 'C17', 'C18', 'C19', 'C20', 'C21', 'user_id',\n       'user_id&media_id', 'user_id&C14_d', 'user_id&C17_d', 'user_id&C14_h',\n       'user_id&C17_h', 'time'],\n      dtype='object')"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 9
    }
   ],
   "source": [
    "train_df.columns"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [],
   "source": [
    "train_lgb = pd.DataFrame(train_df,columns=['click','device_ip','user_id',\n",
    "       'user_id&media_id', 'user_id&C14_d', 'user_id&C17_d', 'user_id&C14_h',\n",
    "       'user_id&C17_h', 'time'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [
    "test_lgb = pd.DataFrame(test_df,columns=['click','device_ip','user_id',\n",
    "       'user_id&media_id', 'user_id&C14_d', 'user_id&C17_d', 'user_id&C14_h',\n",
    "       'user_id&C17_h', 'time'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "data": {
      "text/plain": "   click device_ip            user_id  user_id&media_id  user_id&C14_d  \\\n0      0  ddd2926e  ddd2926e_44956a24                 1              1   \n1      0  96809ac8  96809ac8_711ee120                 1              1   \n2      0  b3cf8def  b3cf8def_8a4875bd                 1              1   \n3      0  e8275b8f  e8275b8f_6332421a                 1              1   \n4      0  9644d0bf  9644d0bf_779d90c2                 1              1   \n\n   user_id&C17_d  user_id&C14_h  user_id&C17_h  time  \n0              1              1              1    -1  \n1              1              1              1    -1  \n2              1              1              1    -1  \n3              1              1              1    -1  \n4              1              1              1    -1  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>click</th>\n      <th>device_ip</th>\n      <th>user_id</th>\n      <th>user_id&amp;media_id</th>\n      <th>user_id&amp;C14_d</th>\n      <th>user_id&amp;C17_d</th>\n      <th>user_id&amp;C14_h</th>\n      <th>user_id&amp;C17_h</th>\n      <th>time</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>ddd2926e</td>\n      <td>ddd2926e_44956a24</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>96809ac8</td>\n      <td>96809ac8_711ee120</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>b3cf8def</td>\n      <td>b3cf8def_8a4875bd</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0</td>\n      <td>e8275b8f</td>\n      <td>e8275b8f_6332421a</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0</td>\n      <td>9644d0bf</td>\n      <td>9644d0bf_779d90c2</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 15
    }
   ],
   "source": [
    "train_lgb.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [],
   "source": [
    "test_lgb['click']=test_lgb['click'].apply(lambda x:-1)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [
    {
     "data": {
      "text/plain": "-1    4577464\nName: click, dtype: int64"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 18
    }
   ],
   "source": [
    "test_lgb['click'].value_counts()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [],
   "source": [
    "train_lgb['click']=train_lgb['click'].replace(0,-1)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [
    {
     "data": {
      "text/plain": "-1    8338643\n 1    1661357\nName: click, dtype: int64"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 21
    }
   ],
   "source": [
    "train_lgb['click'].value_counts()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [],
   "source": [
    "def prep_ip(df):\n",
    "    m = feat['dip_'+df]\n",
    "    return m"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [],
   "source": [
    "train_lgb['device_ip_c']=train_lgb['device_ip'].apply(prep_ip)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "outputs": [
    {
     "data": {
      "text/plain": "1      1144235\n2       745408\n3       540051\n4       414068\n5       330370\n        ...   \n448        448\n427        427\n415        415\n414        414\n397        397\nName: device_ip_c, Length: 1053, dtype: int64"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 26
    }
   ],
   "source": [
    "train_lgb['device_ip_c'].value_counts()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [],
   "source": [
    "test_lgb['device_ip_c']=test_lgb['device_ip'].apply(prep_ip)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [],
   "source": [
    "def prep_uid(df):\n",
    "    m = feat['uid_'+df]\n",
    "    return m"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "outputs": [],
   "source": [
    "train_lgb['user_id_c']=train_lgb['user_id'].apply(prep_uid)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [],
   "source": [
    "test_lgb['user_id_c']=test_lgb['user_id'].apply(prep_uid)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "outputs": [],
   "source": [
    "train_lgb=train_lgb.drop(columns=['device_ip','user_id'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [],
   "source": [
    "test_lgb=test_lgb.drop(columns=['device_ip','user_id'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "outputs": [
    {
     "data": {
      "text/plain": "   click  user_id&media_id  user_id&C14_d  user_id&C17_d  user_id&C14_h  \\\n0     -1                 1              1              1              1   \n1     -1                 1              1              1              1   \n2     -1                 1              1              1              1   \n3     -1                 1              1              1              1   \n4     -1                 1              1              1              1   \n\n   user_id&C17_h  time  device_ip_c  user_id_c  \n0              1    -1        16875         18  \n1              1    -1            7          7  \n2              1    -1            2          2  \n3              1    -1            9          2  \n4              1    -1           31         31  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>click</th>\n      <th>user_id&amp;media_id</th>\n      <th>user_id&amp;C14_d</th>\n      <th>user_id&amp;C17_d</th>\n      <th>user_id&amp;C14_h</th>\n      <th>user_id&amp;C17_h</th>\n      <th>time</th>\n      <th>device_ip_c</th>\n      <th>user_id_c</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n      <td>16875</td>\n      <td>18</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n      <td>7</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>-1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n      <td>2</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>-1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n      <td>9</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>-1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n      <td>31</td>\n      <td>31</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 33
    }
   ],
   "source": [
    "train_lgb.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [
    {
     "data": {
      "text/plain": "   click  user_id&media_id  user_id&C14_d  user_id&C17_d  user_id&C14_h  \\\n0     -1                 1              1              1              1   \n1     -1                 1              1              1              1   \n2     -1                 1              1              1              1   \n3     -1                 1              1              1              1   \n4     -1                 1              1              1              1   \n\n   user_id&C17_h  time  device_ip_c  user_id_c  \n0              1    -1            4          4  \n1              1    -1          508          5  \n2              1    -1           65          7  \n3              1    -1            7          7  \n4              1    -1            1          1  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>click</th>\n      <th>user_id&amp;media_id</th>\n      <th>user_id&amp;C14_d</th>\n      <th>user_id&amp;C17_d</th>\n      <th>user_id&amp;C14_h</th>\n      <th>user_id&amp;C17_h</th>\n      <th>time</th>\n      <th>device_ip_c</th>\n      <th>user_id_c</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n      <td>4</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n      <td>508</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>-1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n      <td>65</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>-1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n      <td>7</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>-1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 24
    }
   ],
   "source": [
    "test_lgb.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "outputs": [],
   "source": [
    "train_lgb.to_csv(\"train_lgb.csv\",index=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [],
   "source": [
    "test_lgb.to_csv(\"test_lgb.csv\",index=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}